import urllib2, httplib, socket, time
from BeautifulSoup import BeautifulSoup

"""
Search the url and returns all anchor links
"""

agent = "Stabber v0.4 http://labs.gorilla3D.com/"

class Anchor:
    def __init__(self, link):
        self.href = link
class Page:
    def __init__(self,url):
        self.url = url
        self.header = None
	self.contentType = None
	self.status = 404
        self.data = ""
        self.soup = "" 
        self.links = []
    def diggLinks(self):
	for i in self.soup.findAll('a'):
		for attr in i.attrs:
			if attr[0] == "href":
				link = Anchor(attr[1])
				self.links.append(link)
            

def search(url):
    global agent
    crawled = []
    try:
        timeout = 10 # timeout in seconds
        socket.setdefaulttimeout(timeout)
        start_ts = time.time()
        headers = None
	
        try:
            req = urllib2.Request(url)
            req.add_header("User-Agent", agent)
            response = urllib2.urlopen(req)
            headers = response.info().headers
            #status = response.info().status
        except urllib2.HTTPError, e:
            print url, "\n\t^----------Not found"
	    return Page(url)
	
        if headers:
		results = response.read()
		contentType = False
		filetype = ""
		for head in headers:
			if head.lower().find('content-type') > -1:
				filetype =  head.lower().split(":")[1]
				if head.lower().find('text/html') > -1:
					contentType = True
		if contentType:
			page 			= Page(url)
			page.header 		= headers
			page.contentType	= filetype
			page.data 		= results
			page.soup 		= BeautifulSoup(page.data)
			page.diggLinks()
			print url #, headers
			return page
			
		else:
			page 			= Page(url)
			page.contentType	= filetype
			print url, "\n\t^----------%s invalid type"%(filetype)
			return page
        else:
		print url, "\n\t^----------Not found"
		#print "\t",headers
		return Page(url)
		
    except IOError:
        return Page(url)
	
    except Exception, msg:
        #print msg
        print url, "\n\t^----------Not found"
	return Page(url)
